{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "567c5ccb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a0a0470e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Imported lazytransform v1.15. \n",
      "\n",
      "Imported featurewiz 0.5.6. Use the following syntax:\n",
      "    >>> wiz = FeatureWiz(feature_engg = '', nrows=None, transform_target=True,\n",
      "        \t\tcategory_encoders=\"auto\", auto_encoders='VAE', ae_options={},\n",
      "        \t\tadd_missing=False, imbalanced=False, verbose=0)\n",
      "    >>> X_train_selected, y_train = wiz.fit_transform(X_train, y_train)\n",
      "    >>> X_test_selected = wiz.transform(X_test)\n",
      "    >>> selected_features = wiz.features\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "from featurewiz import FeatureWiz\n",
    "from featurewiz import print_regression_metrics, print_classification_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "21a19841",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6497, 13)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "      <th>red_wine</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.6</td>\n",
       "      <td>0.098</td>\n",
       "      <td>25.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.9968</td>\n",
       "      <td>3.20</td>\n",
       "      <td>0.68</td>\n",
       "      <td>9.8</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.04</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0.092</td>\n",
       "      <td>15.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.9970</td>\n",
       "      <td>3.26</td>\n",
       "      <td>0.65</td>\n",
       "      <td>9.8</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.2</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.56</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.075</td>\n",
       "      <td>17.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.9980</td>\n",
       "      <td>3.16</td>\n",
       "      <td>0.58</td>\n",
       "      <td>9.8</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0            7.4              0.70         0.00             1.9      0.076   \n",
       "1            7.8              0.88         0.00             2.6      0.098   \n",
       "2            7.8              0.76         0.04             2.3      0.092   \n",
       "3           11.2              0.28         0.56             1.9      0.075   \n",
       "4            7.4              0.70         0.00             1.9      0.076   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \\\n",
       "0                 11.0                  34.0   0.9978  3.51       0.56   \n",
       "1                 25.0                  67.0   0.9968  3.20       0.68   \n",
       "2                 15.0                  54.0   0.9970  3.26       0.65   \n",
       "3                 17.0                  60.0   0.9980  3.16       0.58   \n",
       "4                 11.0                  34.0   0.9978  3.51       0.56   \n",
       "\n",
       "   alcohol  quality  red_wine  \n",
       "0      9.4        5         1  \n",
       "1      9.8        5         1  \n",
       "2      9.8        5         1  \n",
       "3      9.8        6         1  \n",
       "4      9.4        5         1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainfile = 'c:/users/ram/documents/ram/data_sets/kaggle/diabetes.csv'\n",
    "datapath = '../Ram/Data_Sets/'\n",
    "filename = 'winequality.csv'\n",
    "#filename = 'affairs.csv'\n",
    "trainfile = datapath+filename\n",
    "sep = ','\n",
    "dft = pd.read_csv(trainfile,sep=sep)\n",
    "#dft.drop(['affairs','affair'],axis=1, inplace=True)\n",
    "print(dft.shape)\n",
    "dft.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "603bf23c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = 'quality'\n",
    "#target = 'affair_multiclass'\n",
    "modeltype = 'Multi_Classification'\n",
    "preds = [x for x in list(dft) if x not in target]\n",
    "dft[target].nunique()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "1ba875d4",
   "metadata": {},
   "source": [
    "from sklearn.datasets import make_classification, make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, mean_squared_error\n",
    "if modeltype == 'Regression':\n",
    "    X, y = make_regression(n_samples=10000, noise=1000, n_features=8, random_state=0)\n",
    "else:\n",
    "    X, y = make_classification(n_samples=10000, n_classes=5, n_features=8, n_informative=4, random_state=0)\n",
    "# split dataset into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1494931d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5197, 12) (1300, 12)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from featurewiz import FE_kmeans_resampler\n",
    "if modeltype == 'Regression':\n",
    "    X_train, X_test, y_train, y_test = train_test_split(dft[preds], dft[target], test_size=0.20, random_state=1,)\n",
    "    X_train_over, y_train_over = FE_kmeans_resampler(X_train, y_train, target, smote='',verbose=0)\n",
    "    print(X_train_over.shape, X_test.shape)\n",
    "    #train, test = pd.concat([X_train_over, pd.Series(y_train_over,name=target)], axis=1), pd.concat([X_test, y_test], axis=1)\n",
    "    train, test = train_test_split(dft, test_size=0.20, random_state=42)\n",
    "else:\n",
    "    X_train, X_test, y_train, y_test = train_test_split(dft[preds], dft[target], test_size=0.20, \n",
    "                                                    stratify=dft[target],\n",
    "                                                    random_state=42)\n",
    "    train, test = train_test_split(dft, test_size=0.20, random_state=42,\n",
    "                                                    stratify=dft[target]\n",
    "                                                   )\n",
    "print(X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f387d3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "featurewiz is given 0.9 as correlation limit...\n",
      "    Skipping feature engineering since no feature_engg input...\n",
      "    final list of category encoders given: ['onehot', 'label']\n",
      "    You need to pip install tensorflow>= 2.5 in order to use this Autoencoder option.\n",
      "Since Auto Encoders are selected for feature extraction,\n",
      "    Recursive XGBoost is also skipped...\n",
      "CNNAutoEncoder()\n",
      "    AE dictionary given: dict_items([])\n",
      "    final list of scalers given: [minmax]\n"
     ]
    }
   ],
   "source": [
    "scaler = FeatureWiz(feature_engg = '', nrows=None, transform_target=True,\n",
    "        \t\tcategory_encoders=\"auto\", auto_encoders='CNN_ADD', ae_options={},\n",
    "        \t\tadd_missing=False, imbalanced=False, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a8d5fedd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded input data. Shape = (5197, 12)\n",
      "#### Starting featurewiz transform for train data ####\n",
      "    Single_Label Multi_Classification problem \n",
      "Shape of dataset: (5197, 12). Now we classify variables into different types...\n",
      "Time taken to define data pipeline = 1 second(s)\n",
      "No model input given...\n",
      "Lazy Transformer Pipeline created...\n",
      "    transformed target from object type to numeric\n",
      "    Time taken to fit dataset = 1 second(s)\n",
      "    Time taken to transform dataset = 1 second(s)\n",
      "    Shape of transformed dataset: (5197, 12)\n",
      "    No hyperparam selection since GAN or CNN is selected for auto_encoders...\n",
      "Fitting and transforming CNNAutoEncoder for dataset...\n",
      "Epoch 1/100\n",
      "130/130 [==============================] - 2s 7ms/step - loss: 0.0133 - val_loss: 0.0038 - lr: 0.0010\n",
      "Epoch 2/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 0.0029 - val_loss: 0.0022 - lr: 0.0010\n",
      "Epoch 3/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 0.0019 - val_loss: 0.0015 - lr: 0.0010\n",
      "Epoch 4/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 0.0012 - val_loss: 0.0011 - lr: 0.0010\n",
      "Epoch 5/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 9.0416e-04 - val_loss: 8.1538e-04 - lr: 0.0010\n",
      "Epoch 6/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 6.8004e-04 - val_loss: 5.5630e-04 - lr: 0.0010\n",
      "Epoch 7/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 4.7319e-04 - val_loss: 4.5762e-04 - lr: 0.0010\n",
      "Epoch 8/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 4.0537e-04 - val_loss: 4.0449e-04 - lr: 0.0010\n",
      "Epoch 9/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 3.7104e-04 - val_loss: 3.6171e-04 - lr: 0.0010\n",
      "Epoch 10/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 3.4615e-04 - val_loss: 3.7683e-04 - lr: 0.0010\n",
      "Epoch 11/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 3.2180e-04 - val_loss: 3.3386e-04 - lr: 0.0010\n",
      "Epoch 12/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 3.1295e-04 - val_loss: 3.1308e-04 - lr: 0.0010\n",
      "Epoch 13/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 2.9671e-04 - val_loss: 3.3688e-04 - lr: 0.0010\n",
      "Epoch 14/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 2.4843e-04 - val_loss: 2.6806e-04 - lr: 5.0000e-04\n",
      "Epoch 15/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 2.4102e-04 - val_loss: 2.5663e-04 - lr: 5.0000e-04\n",
      "Epoch 16/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 2.3189e-04 - val_loss: 2.6313e-04 - lr: 5.0000e-04\n",
      "Epoch 17/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 2.3202e-04 - val_loss: 2.5388e-04 - lr: 5.0000e-04\n",
      "Epoch 18/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 2.2195e-04 - val_loss: 2.3446e-04 - lr: 5.0000e-04\n",
      "Epoch 19/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 2.1615e-04 - val_loss: 2.3867e-04 - lr: 5.0000e-04\n",
      "Epoch 20/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.9608e-04 - val_loss: 2.2372e-04 - lr: 2.5000e-04\n",
      "Epoch 21/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 1.9390e-04 - val_loss: 2.1614e-04 - lr: 2.5000e-04\n",
      "Epoch 22/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 1.8848e-04 - val_loss: 2.0742e-04 - lr: 2.5000e-04\n",
      "Epoch 23/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.8496e-04 - val_loss: 2.1073e-04 - lr: 2.5000e-04\n",
      "Epoch 24/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.8005e-04 - val_loss: 2.0087e-04 - lr: 2.5000e-04\n",
      "Epoch 25/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.7124e-04 - val_loss: 1.9250e-04 - lr: 1.2500e-04\n",
      "Epoch 26/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.6716e-04 - val_loss: 1.8936e-04 - lr: 1.2500e-04\n",
      "Epoch 27/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.6401e-04 - val_loss: 1.8330e-04 - lr: 1.2500e-04\n",
      "Epoch 28/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.6035e-04 - val_loss: 1.8186e-04 - lr: 1.2500e-04\n",
      "Epoch 29/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.5901e-04 - val_loss: 1.8252e-04 - lr: 1.2500e-04\n",
      "Epoch 30/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 1.5409e-04 - val_loss: 1.7704e-04 - lr: 1.0000e-04\n",
      "Epoch 31/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.5029e-04 - val_loss: 1.6895e-04 - lr: 1.0000e-04\n",
      "Epoch 32/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.4684e-04 - val_loss: 1.6386e-04 - lr: 1.0000e-04\n",
      "Epoch 33/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.4408e-04 - val_loss: 1.6665e-04 - lr: 1.0000e-04\n",
      "Epoch 34/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.4066e-04 - val_loss: 1.5804e-04 - lr: 1.0000e-04\n",
      "Epoch 35/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.3626e-04 - val_loss: 1.5064e-04 - lr: 1.0000e-04\n",
      "Epoch 36/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.3172e-04 - val_loss: 1.4678e-04 - lr: 1.0000e-04\n",
      "Epoch 37/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.2683e-04 - val_loss: 1.4821e-04 - lr: 1.0000e-04\n",
      "Epoch 38/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.2296e-04 - val_loss: 1.3887e-04 - lr: 1.0000e-04\n",
      "Epoch 39/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.1663e-04 - val_loss: 1.2789e-04 - lr: 1.0000e-04\n",
      "Epoch 40/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.1185e-04 - val_loss: 1.2739e-04 - lr: 1.0000e-04\n",
      "Epoch 41/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.0750e-04 - val_loss: 1.1749e-04 - lr: 1.0000e-04\n",
      "Epoch 42/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 1.0330e-04 - val_loss: 1.1409e-04 - lr: 1.0000e-04\n",
      "Epoch 43/100\n",
      "130/130 [==============================] - 1s 7ms/step - loss: 9.8775e-05 - val_loss: 1.0765e-04 - lr: 1.0000e-04\n",
      "Epoch 44/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 9.4918e-05 - val_loss: 1.0886e-04 - lr: 1.0000e-04\n",
      "Epoch 45/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 9.0940e-05 - val_loss: 1.0206e-04 - lr: 1.0000e-04\n",
      "Epoch 46/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 8.8166e-05 - val_loss: 9.9367e-05 - lr: 1.0000e-04\n",
      "Epoch 47/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 8.5611e-05 - val_loss: 9.5914e-05 - lr: 1.0000e-04\n",
      "Epoch 48/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 8.2108e-05 - val_loss: 9.6259e-05 - lr: 1.0000e-04\n",
      "Epoch 49/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 8.0729e-05 - val_loss: 9.1706e-05 - lr: 1.0000e-04\n",
      "Epoch 50/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 7.8945e-05 - val_loss: 8.6837e-05 - lr: 1.0000e-04\n",
      "Epoch 51/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 7.7156e-05 - val_loss: 8.8305e-05 - lr: 1.0000e-04\n",
      "Epoch 52/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 7.5093e-05 - val_loss: 8.8565e-05 - lr: 1.0000e-04\n",
      "Epoch 53/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 7.5755e-05 - val_loss: 8.5557e-05 - lr: 1.0000e-04\n",
      "Epoch 54/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 7.2456e-05 - val_loss: 8.1475e-05 - lr: 1.0000e-04\n",
      "Epoch 55/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 7.1072e-05 - val_loss: 7.8665e-05 - lr: 1.0000e-04\n",
      "Epoch 56/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 7.1659e-05 - val_loss: 8.1373e-05 - lr: 1.0000e-04\n",
      "Epoch 57/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 6.9972e-05 - val_loss: 8.0773e-05 - lr: 1.0000e-04\n",
      "Epoch 58/100\n",
      "130/130 [==============================] - 1s 6ms/step - loss: 6.8407e-05 - val_loss: 7.8222e-05 - lr: 1.0000e-04\n",
      "Epoch 59/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.5964e-05 - val_loss: 7.4572e-05 - lr: 1.0000e-04\n",
      "Epoch 60/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.6795e-05 - val_loss: 8.0847e-05 - lr: 1.0000e-04\n",
      "Epoch 61/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "130/130 [==============================] - 1s 5ms/step - loss: 6.5861e-05 - val_loss: 7.6759e-05 - lr: 1.0000e-04\n",
      "Epoch 62/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.4737e-05 - val_loss: 7.4082e-05 - lr: 1.0000e-04\n",
      "Epoch 63/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.4065e-05 - val_loss: 7.2013e-05 - lr: 1.0000e-04\n",
      "Epoch 64/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.3586e-05 - val_loss: 7.1381e-05 - lr: 1.0000e-04\n",
      "Epoch 65/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.2723e-05 - val_loss: 6.9830e-05 - lr: 1.0000e-04\n",
      "Epoch 66/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.1998e-05 - val_loss: 7.1838e-05 - lr: 1.0000e-04\n",
      "Epoch 67/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.0689e-05 - val_loss: 6.7790e-05 - lr: 1.0000e-04\n",
      "Epoch 68/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.0923e-05 - val_loss: 6.7505e-05 - lr: 1.0000e-04\n",
      "Epoch 69/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 6.0179e-05 - val_loss: 6.8082e-05 - lr: 1.0000e-04\n",
      "Epoch 70/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 5.8860e-05 - val_loss: 7.0600e-05 - lr: 1.0000e-04\n",
      "Epoch 71/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 5.9904e-05 - val_loss: 6.6730e-05 - lr: 1.0000e-04\n",
      "Epoch 72/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 5.7617e-05 - val_loss: 6.6053e-05 - lr: 1.0000e-04\n",
      "Epoch 73/100\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 5.8018e-05 - val_loss: 6.3887e-05 - lr: 1.0000e-04\n",
      "Epoch 74/100\n",
      "129/130 [============================>.] - ETA: 0s - loss: 5.6845e-05Restoring model weights from the end of the best epoch: 64.\n",
      "130/130 [==============================] - 1s 5ms/step - loss: 5.6692e-05 - val_loss: 6.5505e-05 - lr: 1.0000e-04\n",
      "Epoch 00074: early stopping\n",
      "Shape of transformed data due to auto encoder = (5197, 24)\n",
      "    Single_Label Multi_Classification problem \n",
      "Starting SULOV with 24 features...\n",
      "    there are no null values in dataset...\n",
      "    there are no null values in target column...\n",
      "Completed SULOV. 12 features selected\n",
      "    time taken to run entire featurewiz = 57 second(s)\n",
      "Recursive XGBoost selected 12 features...\n"
     ]
    }
   ],
   "source": [
    "# Load and preprocess your dataset\n",
    "# Assuming X_train and y_train are your training data and labels\n",
    "X_train_selected, y_train = scaler.fit_transform(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1d287a58",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#### Starting featurewiz transform for test data ####\n",
      "Loaded input data. Shape = (1300, 12)\n",
      "#### Starting lazytransform for test data ####\n",
      "    Time taken to transform dataset = 1 second(s)\n",
      "    Shape of transformed dataset: (1300, 12)\n",
      "Shape of transformed data due to auto encoder = (1300, 24)\n",
      "Returning dataframe with 12 features \n"
     ]
    }
   ],
   "source": [
    "### Since you modified y_train to numeric, you must do same for y_test\n",
    "X_test_selected = scaler.transform(X_test)\n",
    "if scaler.lazy.yformer:\n",
    "    y_test = scaler.lazy.yformer.transform(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e1a62c58",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bal accu 36%\n",
      "ROC AUC = 0.84\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00         6\n",
      "           1       0.71      0.12      0.20        43\n",
      "           2       0.73      0.69      0.71       428\n",
      "           3       0.65      0.79      0.71       567\n",
      "           4       0.69      0.56      0.62       216\n",
      "           5       0.93      0.36      0.52        39\n",
      "           6       0.00      0.00      0.00         1\n",
      "\n",
      "    accuracy                           0.68      1300\n",
      "   macro avg       0.53      0.36      0.39      1300\n",
      "weighted avg       0.69      0.68      0.67      1300\n",
      "\n",
      "final average balanced accuracy score = 0.36\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
    "from sklearn.utils import class_weight\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from featurewiz import get_class_distribution\n",
    "from xgboost import XGBClassifier, XGBRFRegressor\n",
    "# Updating the Random Forest Classifier with the corrected class weights\n",
    "if modeltype == 'Regression':\n",
    "    #rf_classifier = RandomForestRegressor(random_state=42)\n",
    "    rf_classifier = XGBRFRegressor(n_estimators=300, random_state=99)\n",
    "else:\n",
    "    # Correctly computing class weights for the classes present in the training set\n",
    "    class_weights_dict_corrected = get_class_distribution(y_train)\n",
    "    rf_classifier = RandomForestClassifier(n_estimators=100, class_weight=class_weights_dict_corrected, random_state=42)\n",
    "\n",
    "\n",
    "# Fitting the classifier on the training data\n",
    "rf_classifier.fit(X_train_selected, y_train)\n",
    "\n",
    "# Predicting on the test set\n",
    "y_pred = rf_classifier.predict(X_test_selected)\n",
    "\n",
    "if modeltype == 'Regression':\n",
    "    print_regression_metrics(y_test, y_pred, verbose=1)\n",
    "else:\n",
    "    # Evaluating the classifier\n",
    "    y_probas = rf_classifier.predict_proba(X_test_selected)\n",
    "    print_classification_metrics(y_test, y_pred, y_probas, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dd98599",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d71a2530",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
